{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    @Author: King\\n    @Date: 2019.05.16\\n    @Purpose: Attention-Based-BiLSTM-relation-extraction\\n    @Introduction:  Attention-Based-BiLSTM-relation-extraction\\n    @Datasets: Chinese relation extration datasets\\n    @Link : \\n    @Reference : https://github.com/SeoSangwoo/Attention-Based-BiLSTM-relation-extraction\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# encoding = utf8\n",
    "'''\n",
    "    @Author: King\n",
    "    @Date: 2019.05.16\n",
    "    @Purpose: Attention-Based-BiLSTM-relation-extraction\n",
    "    @Introduction:  Attention-Based-BiLSTM-relation-extraction\n",
    "    @Datasets: Chinese relation extration datasets\n",
    "    @Link : 论文研读笔记作业-https://mp.weixin.qq.com/s?__biz=MzAxMTU5Njg4NQ==&mid=100001827&idx=2&sn=27cd33fa69eaf376a92352f65b293e90\n",
    "    @Reference : https://github.com/SeoSangwoo/Attention-Based-BiLSTM-relation-extraction\n",
    "    @paper ： https://www.aclweb.org/anthology/P16-2034\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification\n",
    "\n",
    "Tensorflow Implementation of Deep Learning Approach for Relation Extraction Challenge(SemEval-2010 Task #8: Multi-Way Classification of Semantic Relations Between Pairs of Nominals) via Attention-based BiLSTM.\n",
    "\n",
    "Original paper [Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification](http://anthology.aclweb.org/P16-2034) \n",
    "\n",
    "<p align=\"center\">\n",
    "\t<img width=\"700\" height=\"400\" src=\"img/Attention-Based-BiLSTM-relation-extraction.png\">\n",
    "</p>\n",
    "\n",
    "### Requrements\n",
    "\n",
    "* Python (>=3.5)\n",
    "\n",
    "* TensorFlow (>=r1.0)\n",
    "\n",
    "* scikit-learn (>=0.18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、Settings Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Settings(object):\n",
    "    def __init__(self):\n",
    "        \n",
    "        '''\n",
    "            Data loading params \n",
    "        '''\n",
    "        ## Path of train data\n",
    "        self.train_path = \"E:/pythonWp/game/CCKS2019/RelationshipExtraction/origin_data/sample10_test_sent.txt\"\n",
    "        # Path of test data\n",
    "        self.test_path = \"E:/pythonWp/game/CCKS2019/RelationshipExtraction/origin_data/sample10_test_sent.txt\"\n",
    "        # Path of relation2id data\n",
    "        self.relation2id_path = \"E:/pythonWp/nlp/relation_extraction/Information-Extraction-Chinese_suss/RE_BGRU_2ATT/origin_data/relation2id.txt\"\n",
    "        # Max sentence length in data\n",
    "        self.max_sentence_length = 90\n",
    "        # Percentage of the training data to use for validation\n",
    "        self.dev_sample_percentage = 0.1\n",
    "        \n",
    "        '''\n",
    "            Model Hyper-parameters \n",
    "        '''\n",
    "        '''\n",
    "            1、Embeddings\n",
    "        '''\n",
    "        # Path of pre-trained word embeddings \n",
    "        self.embedding_path = \"E:/pythonWp/nlp/relation_extraction/Information-Extraction-Chinese_suss/RE_BGRU_2ATT/origin_data/vec_char.txt\"\n",
    "        # Dimensionality of word embedding (default: 100)\n",
    "        self.embedding_dim = 100\n",
    "        # Dropout keep probability of embedding layer (default: 0.7)\n",
    "        self.emb_dropout_keep_prob = 0.7\n",
    "        \n",
    "        '''\n",
    "            2、AttLSTM\n",
    "        '''\n",
    "        # Dimensionality of RNN hidden (default: 100)\n",
    "        self.hidden_size = 100\n",
    "        # Dropout keep probability of RNN (default: 0.7)\n",
    "        self.rnn_dropout_keep_prob = 0.7\n",
    "        \n",
    "        '''\n",
    "            3、Misc\n",
    "        '''\n",
    "        # Description for model\n",
    "        self.desc = \"\"\n",
    "        # Dropout keep probability of RNN (default: 0.7)\n",
    "        self.dropout_keep_prob = 0.5\n",
    "        # L2 regularization lambda (default: 1e-5)\n",
    "        self.l2_reg_lambda = 1e-5\n",
    "        \n",
    "        '''\n",
    "            4、Training parameters\n",
    "        '''\n",
    "        # Description for model\n",
    "        self.batch_size = 10\n",
    "        # Number of training epochs (Default: 100)\n",
    "        self.num_epochs = 5\n",
    "        # Number of iterations to display training information\n",
    "        self.display_every = 5\n",
    "        # Evaluate model on dev set after this many steps (default: 100)\n",
    "        self.evaluate_every = 100\n",
    "        # Number of checkpoints to store (default: 5)\n",
    "        self.num_checkpoints = 5\n",
    "        # Which learning rate to start with (Default: 1.0)\n",
    "        self.learning_rate = 1.0\n",
    "        # Decay rate for learning rate (Default: 0.9)\n",
    "        self.decay_rate = 0.9\n",
    "        \n",
    "        '''\n",
    "            5、Testing parameters\n",
    "        '''\n",
    "        # Checkpoint directory from training run\n",
    "        self.checkpoint_dir = \"\"\n",
    "        \n",
    "        '''\n",
    "            6、Misc Parameters\n",
    "        '''\n",
    "        # Allow device soft device placement\n",
    "        self.allow_soft_placement = True\n",
    "        # Log placement of ops on devices\n",
    "        self.log_device_placement = False\n",
    "        # Allow gpu memory growth\n",
    "        self.gpu_allow_growth = True\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、数据处理模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    工具包 end\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "'''\n",
    "    工具包 begin\n",
    "'''\n",
    "import sys\n",
    "if sys.version_info[0] > 2:\n",
    "    is_py3 = True\n",
    "else:\n",
    "    reload(sys)\n",
    "    sys.setdefaultencoding(\"utf-8\")\n",
    "    is_py3 = False\n",
    "\n",
    "def native_word(word, encoding='utf-8'):\n",
    "    \"\"\"如果在python2下面使用python3训练的模型，可考虑调用此函数转化一下字符编码\"\"\"\n",
    "    if not is_py3:\n",
    "        return word.encode(encoding)\n",
    "    else:\n",
    "        return word\n",
    "\n",
    "def native_content(content):\n",
    "    if not is_py3:\n",
    "        return content.decode('utf-8')\n",
    "    else:\n",
    "        return content\n",
    "\n",
    "def open_file(filename, mode='r'):\n",
    "    \"\"\"\n",
    "    常用文件操作，可在python2和python3间切换.\n",
    "    mode: 'r' or 'w' for read or write\n",
    "    \"\"\"\n",
    "    if is_py3:\n",
    "        return open(filename, mode, encoding='utf-8', errors='ignore')\n",
    "    else:\n",
    "        return open(filename, mode)\n",
    "\n",
    "'''\n",
    "    工具包 end\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取 relation2id 文件数据\n",
    "def load_relation2id_file_cn(filename,demo_flag = False):\n",
    "    '''\n",
    "    读取 data 文件数据\n",
    "    :param filename:    String 文件名称包含路径\n",
    "    :param demo_flag:   String True 只读取 1000 样本数据，Fasle 读取全部数据\n",
    "    :return:\n",
    "        relation2id:   dict    relation to id\n",
    "        id2relation:   list    id to relation \n",
    "    '''\n",
    "    contents_num = 0\n",
    "    relation2id = {}\n",
    "    id2relation = []\n",
    "    with open_file(filename) as f:\n",
    "        for line in f:\n",
    "            try:\n",
    "                data_list = line.replace(\"\\n\",\"\").split(\"\\t\")\n",
    "                relation2id[data_list[0]] = int(data_list[1])\n",
    "                id2relation.append(data_list[0])\n",
    "                contents_num = contents_num + 1\n",
    "                if demo_flag and contents_num == 500:\n",
    "                    break\n",
    "            except:\n",
    "                pass\n",
    "    return relation2id,id2relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取 训练集 文件数据\n",
    "def load_data_and_labels_cn(path,settings):\n",
    "    relation2id,id2relation=load_relation2id_file_cn(filename=settings.relation2id_path)   \n",
    "    data = []\n",
    "\n",
    "    df = pd.read_csv(path,quoting = 3, sep='\\t',names=['e1','e2','r','s'])\n",
    "    ''' print(\"df:{0}\".format(df.iloc[0:2]))\n",
    "        output:\n",
    "            df:    \n",
    "            e1  e2    r                                                  s\n",
    "            0  李烈钧  王侃  NaN              李烈钧加入同盟会:光绪三十三年（1907年），经张断、王侃介绍加入同盟会。\n",
    "            1   陈尸  孔子  NaN  子服景伯把这件事告诉给孔子，并且说：“季孙氏已经被公伯寮迷惑了，我的力量能够把公伯寮杀了，把...\n",
    "    '''\n",
    "    max_sentence_length = 0\n",
    "    sentence_list = df['s'].tolist()\n",
    "    for i in range(0,len(sentence_list)):\n",
    "        tokens = str(sentence_list[i])\n",
    "        if max_sentence_length < len(tokens):\n",
    "            max_sentence_length = len(tokens)\n",
    "        sentence_list[i] = \" \".join(tokens)\n",
    "    ''' print(\"sentence_list:{0}\".format(sentence_list[0:2]))\n",
    "        print(\"max sentence length = {}\\n\".format(max_sentence_length))\n",
    "        print(\"df:{0}\".format(df.iloc[0:2]))\n",
    "        output\n",
    "            sentence_list:['李 烈 钧 加 入 同 盟 会 : 光 绪 三 十 三 年 （ 1 9 0 7 年 ） ， 经 张 断 、 王 侃 介 绍 加 入 同 盟 会  。', '子 服 景 伯 把 这 件 事 告 诉 给 孔 子 ， 并 且 说 ： “ 季 孙 氏 已 经 被 公 伯 寮 迷 惑 了 ， 我 的 力 量 能 够 把 公 伯 寮 杀 了 ， 把 他 陈 尸 于 市 。 ”']\n",
    "            max sentence length = 19751\n",
    "\n",
    "            df:    e1  e2    r                                                  s\n",
    "            0  李烈钧  王侃  NaN              李烈钧加入同盟会:光绪三十三年（1907年），经张断、王侃介绍加入同盟会。\n",
    "            1   陈尸  孔子  NaN  子服景伯把这件事告诉给孔子，并且说：“季孙氏已经被公伯寮迷惑了，我的力量能够把公伯寮杀了，把...\n",
    "    '''\n",
    "\n",
    "    df = df.fillna('NA')            # 将省缺值用 ‘NAN’ 代替\n",
    "\n",
    "    df['label'] = [relation2id[str(r)] for r in df['r']]\n",
    "    ''' print(\"df:{0}\".format(df.iloc[0:2]))\n",
    "        output:\n",
    "            df:    \n",
    "                    e1  e2   r                                                  s  label\n",
    "            0  李烈钧  王侃  NA              李烈钧加入同盟会:光绪三十三年（1907年），经张断、王侃介绍加入同盟会。      0\n",
    "            1   陈尸  孔子  NA  子服景伯把这件事告诉给孔子，并且说：“季孙氏已经被公伯寮迷惑了，我的力量能够把公伯寮杀了，把...      0\n",
    "    '''\n",
    "\n",
    "    # Text Data\n",
    "    x_text = sentence_list\n",
    "    ''' print(\"x_text:{0}\".format(x_text[0:1]))\n",
    "        sys.exit(0)\n",
    "        output:\n",
    "            x_text:['李 烈 钧 加 入 同 盟 会 : 光 绪 三 十 三 年 （ 1 9 0 7 年 ） ， 经 张 断 、 王 侃 介 绍 加 入 同 盟 会 。', '子 服 景 伯 把 这 件 事 告 诉 给 孔 子 ， 并 且 说 ： “ 季 孙 氏 已 经 被 公 伯 寮 迷 惑 了 ， 我 的 力 量 能 够 把 公 伯 寮 杀 了 ， 把 他 陈 尸 于 市 。 ”', '剪 辑 ： 朱 小 勤 、 苏 鸿 文', '区 域 创 新 体 系 的 若 干 文 献 综 述 （ 陈 广 胜 许 小 忠 徐 燕 椿 ）', '在 拍 摄 间 隙 的 时 候 ， 谭 松 韵 与 郭 俊 辰 经 常 一 起 吃 辣 条 。']\n",
    "    '''\n",
    "    # Label Data\n",
    "    y = df['label']\n",
    "    labels_flat = y.values.ravel()\n",
    "    labels_count = np.unique(labels_flat).shape[0]\n",
    "    print(\"labels_flat:{0}\".format(labels_flat))\n",
    "    print(\"labels_count:{0}\".format(labels_count))\n",
    "    ''' print(\"labels_flat:{0}\".format(labels_flat))\n",
    "        print(\"labels_count:{0}\".format(labels_count))\n",
    "        output:\n",
    "            labels_flat:[ 0  0  0 ...  0 29 29]\n",
    "            labels_count:35\n",
    "    '''\n",
    "\n",
    "    # convert class labels from scalars to one-hot vectors\n",
    "    # 0  => [1 0 0 0 0 ... 0 0 0 0 0]\n",
    "    # 1  => [0 1 0 0 0 ... 0 0 0 0 0]\n",
    "    # ...\n",
    "    # 18 => [0 0 0 0 0 ... 0 0 0 0 1]\n",
    "    def dense_to_one_hot(labels_dense, num_classes):\n",
    "        num_labels = labels_dense.shape[0]\n",
    "        index_offset = np.arange(num_labels) * num_classes\n",
    "        labels_one_hot = np.zeros((num_labels, num_classes))\n",
    "        labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "        return labels_one_hot\n",
    "\n",
    "    labels = dense_to_one_hot(labels_flat, labels_count)\n",
    "    labels = labels.astype(np.uint8)\n",
    "    ''' print(\"x_text:{0}\".format(x_text[0:1]))\n",
    "        print(\"labels:{0}\".format(labels[0:1]))\n",
    "        print(\"len(x_text):{0}\".format(len(x_text)))\n",
    "        print(\"len(labels):{0}\".format(len(labels)))\n",
    "        sys.exit(0)\n",
    "        output:\n",
    "            x_text:['李 烈 钧 加 入 同 盟 会 : 光 绪 三 十 三 年 （ 1 9 0 7 年 ） ， 经 张 断 、 王 侃 介 绍 加 入 同 盟 会 。', '子 服 景 伯 把 这 件 事 告 诉 给 孔 子 ， 并 且 说 ： “ 季 孙 氏 已 经 被 公 伯 寮 迷 惑 了 ， 我 的 力 量 能 够 把 公 伯 寮 杀 了 ， 把 他 陈 尸 于 市 。 ”', '剪 辑 ： 朱 小 勤 、 苏 鸿 文', '区 域 创 新 体 系 的 若 干 文 献 综 述 （ 陈 广 胜 许 小 忠 徐 燕 椿 ）', '在 拍 摄 间 隙 的 时 候 ， 谭 松 韵 与 郭 俊 辰 经 常 一 起 吃 辣 条 。']\n",
    "            labels:[[1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
    "             [1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
    "             [1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
    "             [1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
    "             [1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]]\n",
    "            len(x_text):37637\n",
    "            len(labels):37637\n",
    "    '''\n",
    "    return x_text, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labels_flat:[10  0  0 ...  0  0  0]\n",
      "labels_count:35\n",
      "\n"
     ]
    }
   ],
   "source": [
    "settings =Settings()\n",
    "load_data_and_labels_cn(path=settings.train_path,settings=settings)\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_iter(data, batch_size, num_epochs, shuffle=True):\n",
    "    \"\"\"\n",
    "    Generates a batch iterator for a dataset.\n",
    "    \"\"\"\n",
    "    data = np.array(data)\n",
    "    data_size = len(data)\n",
    "    num_batches_per_epoch = int((len(data) - 1) / batch_size) + 1\n",
    "    for epoch in range(num_epochs):\n",
    "        # Shuffle the data at each epoch\n",
    "        if shuffle:\n",
    "            shuffle_indices = np.random.permutation(np.arange(data_size))\n",
    "            shuffled_data = data[shuffle_indices]\n",
    "        else:\n",
    "            shuffled_data = data\n",
    "        for batch_num in range(num_batches_per_epoch):\n",
    "            start_index = batch_num * batch_size\n",
    "            end_index = min((batch_num + 1) * batch_size, data_size)\n",
    "            yield shuffled_data[start_index:end_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3、Attention  Based BiLSTM relation extraction 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\progrom\\python\\python\\python3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# 导入库\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime\n",
    "import time\n",
    "from sklearn.metrics import f1_score\n",
    "import warnings\n",
    "import sklearn.exceptions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  3.1 attention模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention(inputs):\n",
    "    # Trainable parameters\n",
    "    hidden_size = inputs.shape[2].value\n",
    "    u_omega = tf.get_variable(\"u_omega\", [hidden_size], initializer=tf.keras.initializers.glorot_normal())\n",
    "\n",
    "    with tf.name_scope('v'):\n",
    "        v = tf.tanh(inputs)\n",
    "\n",
    "    # For each of the timestamps its vector of size A from `v` is reduced with `u` vector\n",
    "    vu = tf.tensordot(v, u_omega, axes=1, name='vu')  # (B,T) shape\n",
    "    alphas = tf.nn.softmax(vu, name='alphas')  # (B,T) shape\n",
    "\n",
    "    # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape\n",
    "    output = tf.reduce_sum(inputs * tf.expand_dims(alphas, -1), 1)\n",
    "\n",
    "    # Final output with tanh\n",
    "    output = tf.tanh(output)\n",
    "\n",
    "    return output, alphas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2 AttLSTM 模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttLSTM:\n",
    "    def __init__(self, sequence_length, num_classes, vocab_size, embedding_size,\n",
    "                 hidden_size, l2_reg_lambda=0.0):\n",
    "        # Placeholders for input, output and dropout\n",
    "        self.input_text = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_text')\n",
    "        self.input_y = tf.placeholder(tf.float32, shape=[None, num_classes], name='input_y')\n",
    "        self.emb_dropout_keep_prob = tf.placeholder(tf.float32, name='emb_dropout_keep_prob')\n",
    "        self.rnn_dropout_keep_prob = tf.placeholder(tf.float32, name='rnn_dropout_keep_prob')\n",
    "        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')\n",
    "\n",
    "        initializer = tf.keras.initializers.glorot_normal\n",
    "\n",
    "        # Word Embedding Layer\n",
    "        with tf.device('/cpu:0'), tf.variable_scope(\"word-embeddings\"):\n",
    "            self.W_text = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -0.25, 0.25), name=\"W_text\")\n",
    "            self.embedded_chars = tf.nn.embedding_lookup(self.W_text, self.input_text)\n",
    "\n",
    "        # Dropout for Word Embedding\n",
    "        with tf.variable_scope('dropout-embeddings'):\n",
    "            self.embedded_chars = tf.nn.dropout(self.embedded_chars, self.emb_dropout_keep_prob)\n",
    "\n",
    "        # Bidirectional LSTM\n",
    "        with tf.variable_scope(\"bi-lstm\"):\n",
    "            _fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size, initializer=initializer())\n",
    "            fw_cell = tf.nn.rnn_cell.DropoutWrapper(_fw_cell, self.rnn_dropout_keep_prob)\n",
    "            _bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size, initializer=initializer())\n",
    "            bw_cell = tf.nn.rnn_cell.DropoutWrapper(_bw_cell, self.rnn_dropout_keep_prob)\n",
    "            self.rnn_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell,\n",
    "                                                                  cell_bw=bw_cell,\n",
    "                                                                  inputs=self.embedded_chars,\n",
    "                                                                  sequence_length=self._length(self.input_text),\n",
    "                                                                  dtype=tf.float32)\n",
    "            self.rnn_outputs = tf.add(self.rnn_outputs[0], self.rnn_outputs[1])\n",
    "\n",
    "        # Attention\n",
    "        with tf.variable_scope('attention'):\n",
    "            self.attn, self.alphas = attention(self.rnn_outputs)\n",
    "\n",
    "        # Dropout\n",
    "        with tf.variable_scope('dropout'):\n",
    "            self.h_drop = tf.nn.dropout(self.attn, self.dropout_keep_prob)\n",
    "\n",
    "        # Fully connected layer\n",
    "        with tf.variable_scope('output'):\n",
    "            self.logits = tf.layers.dense(self.h_drop, num_classes, kernel_initializer=initializer())\n",
    "            self.predictions = tf.argmax(self.logits, 1, name=\"predictions\")\n",
    "\n",
    "        # Calculate mean cross-entropy loss\n",
    "        with tf.variable_scope(\"loss\"):\n",
    "            losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)\n",
    "            self.l2 = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()])\n",
    "            self.loss = tf.reduce_mean(losses) + l2_reg_lambda * self.l2\n",
    "\n",
    "        # Accuracy\n",
    "        with tf.variable_scope(\"accuracy\"):\n",
    "            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name=\"accuracy\")\n",
    "\n",
    "    # Length of the sequence data\n",
    "    @staticmethod\n",
    "    def _length(seq):\n",
    "        relevant = tf.sign(tf.abs(seq))\n",
    "        length = tf.reduce_sum(relevant, reduction_indices=1)\n",
    "        length = tf.cast(length, tf.int32)\n",
    "        return length\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4、词嵌入加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_word2vec(embedding_path, embedding_dim, vocab):\n",
    "    # initial matrix with random uniform\n",
    "    initW = np.random.randn(len(vocab.vocabulary_), embedding_dim).astype(np.float32) / np.sqrt(len(vocab.vocabulary_))\n",
    "    # load any vectors from the word2vec\n",
    "    print(\"Load glove file {0}\".format(embedding_path))\n",
    "    f = open(embedding_path, 'r', encoding='utf8')\n",
    "    for line in f:\n",
    "        splitLine = line.split(' ')\n",
    "        word = splitLine[0]\n",
    "        embedding = np.asarray(splitLine[1:], dtype='float32')\n",
    "        idx = vocab.vocabulary_.get(word)\n",
    "        if idx != 0:\n",
    "            initW[idx] = embedding\n",
    "    return initW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5、模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=sklearn.exceptions.UndefinedMetricWarning)\n",
    "settings =Settings()\n",
    "\n",
    "def train(settings):\n",
    "    with tf.device('/cpu:0'):\n",
    "        x_text, y = load_data_and_labels_cn(path=settings.train_path,settings=settings)\n",
    "\n",
    "\n",
    "    vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor(settings.max_sentence_length)\n",
    "    x = np.array(list(vocab_processor.fit_transform(x_text)))\n",
    "    print(\"Text Vocabulary Size: {:d}\".format(len(vocab_processor.vocabulary_)))\n",
    "    print(\"x = {0}\".format(x.shape))\n",
    "    print(\"y = {0}\".format(y.shape))\n",
    "    \n",
    "    # Randomly shuffle data to split into train and test(dev)\n",
    "    np.random.seed(10)\n",
    "    shuffle_indices = np.random.permutation(np.arange(len(y)))\n",
    "    x_shuffled = x[shuffle_indices]\n",
    "    y_shuffled = y[shuffle_indices]\n",
    "\n",
    "    # Split train/test set\n",
    "    # TODO: This is very crude, should use cross-validation\n",
    "    dev_sample_index = -1 * int(settings.dev_sample_percentage * float(len(y)))\n",
    "    x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]\n",
    "    y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]\n",
    "    \n",
    "    \n",
    "    with tf.Graph().as_default():\n",
    "        # GPU 配置项\n",
    "        session_conf = tf.ConfigProto(\n",
    "            allow_soft_placement=settings.allow_soft_placement,\n",
    "            log_device_placement=settings.log_device_placement)\n",
    "        session_conf.gpu_options.allow_growth = settings.gpu_allow_growth\n",
    "\n",
    "        sess = tf.Session(config=session_conf)\n",
    "        with sess.as_default():\n",
    "            model = AttLSTM(\n",
    "                sequence_length=x_train.shape[1],\n",
    "                num_classes=y_train.shape[1],\n",
    "                vocab_size=len(vocab_processor.vocabulary_),\n",
    "                embedding_size=settings.embedding_dim,\n",
    "                hidden_size=settings.hidden_size,\n",
    "                l2_reg_lambda=settings.l2_reg_lambda)\n",
    "\n",
    "            # Define Training procedure\n",
    "            global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "            optimizer = tf.train.AdadeltaOptimizer(settings.learning_rate, settings.decay_rate, 1e-6)\n",
    "            gvs = optimizer.compute_gradients(model.loss)\n",
    "            capped_gvs = [(tf.clip_by_value(grad, -1.0, 1.0), var) for grad, var in gvs]\n",
    "            train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step)\n",
    "\n",
    "            # Output directory for models and summaries\n",
    "            timestamp = str(int(time.time()))\n",
    "            out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", timestamp))\n",
    "            print(\"Writing to {}\\n\".format(out_dir))\n",
    "\n",
    "            # Summaries for loss and accuracy\n",
    "            loss_summary = tf.summary.scalar(\"loss\", model.loss)\n",
    "            acc_summary = tf.summary.scalar(\"accuracy\", model.accuracy)\n",
    "\n",
    "            # Train Summaries\n",
    "            train_summary_op = tf.summary.merge([loss_summary, acc_summary])\n",
    "            train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\")\n",
    "            train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)\n",
    "\n",
    "            # Dev summaries\n",
    "            dev_summary_op = tf.summary.merge([loss_summary, acc_summary])\n",
    "            dev_summary_dir = os.path.join(out_dir, \"summaries\", \"dev\")\n",
    "            dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)\n",
    "\n",
    "            # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it\n",
    "            checkpoint_dir = os.path.abspath(os.path.join(out_dir, \"checkpoints\"))\n",
    "            checkpoint_prefix = os.path.join(checkpoint_dir, \"model\")\n",
    "            if not os.path.exists(checkpoint_dir):\n",
    "                os.makedirs(checkpoint_dir)\n",
    "            # 用于保存模型\n",
    "            saver = tf.train.Saver(tf.global_variables(), max_to_keep=settings.num_checkpoints)\n",
    "\n",
    "            # Write vocabulary\n",
    "            vocab_processor.save(os.path.join(out_dir, \"vocab\"))\n",
    "\n",
    "            # Initialize all variables\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "            # Pre-trained word2vec\n",
    "            if settings.embedding_path:\n",
    "                pretrain_W = load_word2vec(settings.embedding_path, settings.embedding_dim, vocab_processor)\n",
    "                sess.run(model.W_text.assign(pretrain_W))\n",
    "                print(\"Success to load pre-trained word2vec model!\\n\")\n",
    "\n",
    "            # Generate batches\n",
    "            batches = batch_iter(list(zip(x_train, y_train)), settings.batch_size, settings.num_epochs)\n",
    "            # Training loop. For each batch...\n",
    "            best_f1 = 0.0  # For save checkpoint(model)\n",
    "            for batch in batches:\n",
    "                x_batch, y_batch = zip(*batch)\n",
    "                # Train\n",
    "                feed_dict = {\n",
    "                    model.input_text: x_batch,\n",
    "                    model.input_y: y_batch,\n",
    "                    model.emb_dropout_keep_prob: settings.emb_dropout_keep_prob,\n",
    "                    model.rnn_dropout_keep_prob: settings.rnn_dropout_keep_prob,\n",
    "                    model.dropout_keep_prob: settings.dropout_keep_prob\n",
    "                }\n",
    "                _, step, summaries, loss, accuracy = sess.run(\n",
    "                    [train_op, global_step, train_summary_op, model.loss, model.accuracy], feed_dict)\n",
    "                train_summary_writer.add_summary(summaries, step)\n",
    "\n",
    "                # Training log display\n",
    "                if step % settings.display_every == 0:\n",
    "                    time_str = datetime.datetime.now().isoformat()\n",
    "                    print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "\n",
    "                # Evaluation\n",
    "                if step % settings.evaluate_every == 0:\n",
    "                    print(\"\\nEvaluation:\")\n",
    "                    feed_dict = {\n",
    "                        model.input_text: x_dev,\n",
    "                        model.input_y: y_dev,\n",
    "                        model.emb_dropout_keep_prob: 1.0,\n",
    "                        model.rnn_dropout_keep_prob: 1.0,\n",
    "                        model.dropout_keep_prob: 1.0\n",
    "                    }\n",
    "                    summaries, loss, accuracy, predictions = sess.run(\n",
    "                        [dev_summary_op, model.loss, model.accuracy, model.predictions], feed_dict)\n",
    "                    dev_summary_writer.add_summary(summaries, step)\n",
    "\n",
    "                    time_str = datetime.datetime.now().isoformat()\n",
    "                    f1 = f1_score(np.argmax(y_dev, axis=1), predictions, labels=np.array(range(1, 35)), average=\"macro\")\n",
    "                    print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "                    print(\"[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): {:g}\\n\".format(f1))\n",
    "\n",
    "                    # Model checkpoint\n",
    "                    if best_f1 < f1:\n",
    "                        best_f1 = f1\n",
    "                        path = saver.save(sess, checkpoint_prefix + \"-{:.3g}\".format(best_f1), global_step=step)\n",
    "                        print(\"Saved model checkpoint to {}\\n\".format(path))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labels_flat:[10  0  0 ...  0  0  0]\n",
      "labels_count:35\n",
      "WARNING:tensorflow:From <ipython-input-13-83814e47bb63>:9: VocabularyProcessor.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From d:\\progrom\\python\\python\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\preprocessing\\text.py:154: CategoricalVocabulary.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.categorical_vocabulary) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From d:\\progrom\\python\\python\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\preprocessing\\text.py:170: tokenizer (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "Text Vocabulary Size: 4022\n",
      "x = (4818, 90)\n",
      "y = (4818, 35)\n",
      "Writing to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\n",
      "\n",
      "Load glove file E:/pythonWp/nlp/relation_extraction/Information-Extraction-Chinese_suss/RE_BGRU_2ATT/origin_data/vec_char.txt\n",
      "Success to load pre-trained word2vec model!\n",
      "\n",
      "2019-05-18T15:40:59.368118: step 5, loss 3.28395, acc 0.8\n",
      "2019-05-18T15:40:59.631307: step 10, loss 1.65106, acc 0.8\n",
      "2019-05-18T15:40:59.911558: step 15, loss 2.82888, acc 0.5\n",
      "2019-05-18T15:41:00.192807: step 20, loss 1.62322, acc 0.7\n",
      "2019-05-18T15:41:00.483292: step 25, loss 1.9708, acc 0.6\n",
      "2019-05-18T15:41:00.754233: step 30, loss 1.45214, acc 0.8\n",
      "2019-05-18T15:41:01.050234: step 35, loss 1.15065, acc 0.8\n",
      "2019-05-18T15:41:01.346902: step 40, loss 0.296738, acc 1\n",
      "2019-05-18T15:41:01.616986: step 45, loss 2.13953, acc 0.6\n",
      "2019-05-18T15:41:01.888502: step 50, loss 0.622795, acc 0.9\n",
      "2019-05-18T15:41:02.176574: step 55, loss 1.02976, acc 0.9\n",
      "2019-05-18T15:41:02.477871: step 60, loss 1.02144, acc 0.8\n",
      "2019-05-18T15:41:02.753073: step 65, loss 0.717639, acc 0.9\n",
      "2019-05-18T15:41:03.037539: step 70, loss 0.419046, acc 1\n",
      "2019-05-18T15:41:03.284702: step 75, loss 0.875383, acc 0.9\n",
      "2019-05-18T15:41:03.539337: step 80, loss 1.45306, acc 0.6\n",
      "2019-05-18T15:41:03.790665: step 85, loss 1.03212, acc 0.8\n",
      "2019-05-18T15:41:04.044510: step 90, loss 1.44995, acc 0.8\n",
      "2019-05-18T15:41:04.338741: step 95, loss 0.980413, acc 0.8\n",
      "2019-05-18T15:41:04.596221: step 100, loss 2.28143, acc 0.5\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:05.197289: step 100, loss 1.2058, acc 0.781705\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0\n",
      "\n",
      "2019-05-18T15:41:05.433694: step 105, loss 1.26096, acc 0.7\n",
      "2019-05-18T15:41:05.689006: step 110, loss 1.64731, acc 0.7\n",
      "2019-05-18T15:41:05.919655: step 115, loss 1.27099, acc 0.8\n",
      "2019-05-18T15:41:06.148087: step 120, loss 1.13084, acc 0.8\n",
      "2019-05-18T15:41:06.381042: step 125, loss 0.940679, acc 0.9\n",
      "2019-05-18T15:41:06.601877: step 130, loss 1.70163, acc 0.6\n",
      "2019-05-18T15:41:06.843299: step 135, loss 2.0827, acc 0.7\n",
      "2019-05-18T15:41:07.076501: step 140, loss 1.28206, acc 0.8\n",
      "2019-05-18T15:41:07.299934: step 145, loss 1.29325, acc 0.8\n",
      "2019-05-18T15:41:07.538732: step 150, loss 0.687137, acc 0.9\n",
      "2019-05-18T15:41:07.775783: step 155, loss 0.531082, acc 0.9\n",
      "2019-05-18T15:41:08.012678: step 160, loss 0.834471, acc 0.9\n",
      "2019-05-18T15:41:08.253921: step 165, loss 1.77513, acc 0.6\n",
      "2019-05-18T15:41:08.489131: step 170, loss 1.95726, acc 0.6\n",
      "2019-05-18T15:41:08.723838: step 175, loss 0.219805, acc 1\n",
      "2019-05-18T15:41:08.947385: step 180, loss 0.540142, acc 0.9\n",
      "2019-05-18T15:41:09.170674: step 185, loss 1.30983, acc 0.8\n",
      "2019-05-18T15:41:09.400806: step 190, loss 0.319813, acc 1\n",
      "2019-05-18T15:41:09.622260: step 195, loss 1.58943, acc 0.7\n",
      "2019-05-18T15:41:09.847327: step 200, loss 0.8144, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:10.201284: step 200, loss 1.10997, acc 0.779626\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0\n",
      "\n",
      "2019-05-18T15:41:10.421519: step 205, loss 1.31378, acc 0.7\n",
      "2019-05-18T15:41:10.656318: step 210, loss 1.3345, acc 0.7\n",
      "2019-05-18T15:41:10.880767: step 215, loss 0.920324, acc 0.8\n",
      "2019-05-18T15:41:11.117361: step 220, loss 1.1535, acc 0.8\n",
      "2019-05-18T15:41:11.334199: step 225, loss 0.662545, acc 0.9\n",
      "2019-05-18T15:41:11.568300: step 230, loss 1.94619, acc 0.5\n",
      "2019-05-18T15:41:11.794364: step 235, loss 1.41767, acc 0.7\n",
      "2019-05-18T15:41:12.034655: step 240, loss 1.95883, acc 0.6\n",
      "2019-05-18T15:41:12.271375: step 245, loss 1.46487, acc 0.8\n",
      "2019-05-18T15:41:12.491849: step 250, loss 1.40212, acc 0.7\n",
      "2019-05-18T15:41:12.732705: step 255, loss 1.03015, acc 0.8\n",
      "2019-05-18T15:41:12.956956: step 260, loss 2.03545, acc 0.7\n",
      "2019-05-18T15:41:13.173707: step 265, loss 1.67122, acc 0.7\n",
      "2019-05-18T15:41:13.411541: step 270, loss 1.04891, acc 0.9\n",
      "2019-05-18T15:41:13.653658: step 275, loss 1.46084, acc 0.6\n",
      "2019-05-18T15:41:13.895762: step 280, loss 1.32246, acc 0.6\n",
      "2019-05-18T15:41:14.135551: step 285, loss 1.58622, acc 0.8\n",
      "2019-05-18T15:41:14.384051: step 290, loss 1.18786, acc 0.7\n",
      "2019-05-18T15:41:14.640380: step 295, loss 2.33982, acc 0.6\n",
      "2019-05-18T15:41:14.874966: step 300, loss 0.456566, acc 0.9\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:15.231416: step 300, loss 1.04652, acc 0.794179\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0134564\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0135-300\n",
      "\n",
      "2019-05-18T15:41:15.956132: step 305, loss 0.656849, acc 0.9\n",
      "2019-05-18T15:41:16.192100: step 310, loss 1.55192, acc 0.7\n",
      "2019-05-18T15:41:16.453389: step 315, loss 1.13587, acc 0.9\n",
      "2019-05-18T15:41:16.699886: step 320, loss 1.22292, acc 0.7\n",
      "2019-05-18T15:41:16.942209: step 325, loss 0.727587, acc 0.9\n",
      "2019-05-18T15:41:17.194053: step 330, loss 1.74477, acc 0.6\n",
      "2019-05-18T15:41:17.444407: step 335, loss 1.29427, acc 0.7\n",
      "2019-05-18T15:41:17.710343: step 340, loss 1.15068, acc 0.8\n",
      "2019-05-18T15:41:17.943490: step 345, loss 1.00885, acc 0.8\n",
      "2019-05-18T15:41:18.175554: step 350, loss 1.08334, acc 0.7\n",
      "2019-05-18T15:41:18.426665: step 355, loss 0.689905, acc 0.8\n",
      "2019-05-18T15:41:18.672009: step 360, loss 0.472229, acc 0.9\n",
      "2019-05-18T15:41:18.917832: step 365, loss 1.68798, acc 0.6\n",
      "2019-05-18T15:41:19.144807: step 370, loss 1.35589, acc 0.8\n",
      "2019-05-18T15:41:19.377186: step 375, loss 1.30505, acc 0.7\n",
      "2019-05-18T15:41:19.619538: step 380, loss 1.01979, acc 0.9\n",
      "2019-05-18T15:41:19.875400: step 385, loss 1.17323, acc 0.8\n",
      "2019-05-18T15:41:20.106122: step 390, loss 0.772854, acc 0.8\n",
      "2019-05-18T15:41:20.347036: step 395, loss 1.54233, acc 0.6\n",
      "2019-05-18T15:41:20.596372: step 400, loss 0.797997, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:20.987806: step 400, loss 0.99458, acc 0.790021\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.00791855\n",
      "\n",
      "2019-05-18T15:41:21.215900: step 405, loss 1.46549, acc 0.8\n",
      "2019-05-18T15:41:21.466471: step 410, loss 0.634291, acc 0.9\n",
      "2019-05-18T15:41:21.713651: step 415, loss 0.231608, acc 1\n",
      "2019-05-18T15:41:21.969286: step 420, loss 1.61236, acc 0.7\n",
      "2019-05-18T15:41:22.245793: step 425, loss 1.04666, acc 0.7\n",
      "2019-05-18T15:41:22.522919: step 430, loss 1.06536, acc 0.8\n",
      "2019-05-18T15:41:22.782225: step 435, loss 0.881996, acc 0.8\n",
      "2019-05-18T15:41:23.037671: step 440, loss 0.633507, acc 0.9\n",
      "2019-05-18T15:41:23.315918: step 445, loss 1.23493, acc 0.9\n",
      "2019-05-18T15:41:23.555278: step 450, loss 1.88104, acc 0.6\n",
      "2019-05-18T15:41:23.795635: step 455, loss 0.704481, acc 0.9\n",
      "2019-05-18T15:41:24.054466: step 460, loss 0.485462, acc 0.9\n",
      "2019-05-18T15:41:24.297524: step 465, loss 0.884123, acc 0.8\n",
      "2019-05-18T15:41:24.542411: step 470, loss 0.951922, acc 0.8\n",
      "2019-05-18T15:41:24.776243: step 475, loss 1.433, acc 0.7\n",
      "2019-05-18T15:41:25.036219: step 480, loss 0.743667, acc 0.9\n",
      "2019-05-18T15:41:25.273221: step 485, loss 0.666718, acc 0.8\n",
      "2019-05-18T15:41:25.519413: step 490, loss 1.08571, acc 0.7\n",
      "2019-05-18T15:41:25.780716: step 495, loss 0.594905, acc 0.8\n",
      "2019-05-18T15:41:26.050992: step 500, loss 0.724155, acc 0.9\n",
      "\n",
      "Evaluation:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-05-18T15:41:26.458361: step 500, loss 0.998784, acc 0.796258\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0151579\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0152-500\n",
      "\n",
      "2019-05-18T15:41:27.354337: step 505, loss 0.442802, acc 0.9\n",
      "2019-05-18T15:41:27.598686: step 510, loss 1.12604, acc 0.8\n",
      "2019-05-18T15:41:27.830575: step 515, loss 0.604998, acc 0.9\n",
      "2019-05-18T15:41:28.073789: step 520, loss 0.742013, acc 0.8\n",
      "2019-05-18T15:41:28.329289: step 525, loss 1.47512, acc 0.8\n",
      "2019-05-18T15:41:28.582612: step 530, loss 1.89703, acc 0.6\n",
      "2019-05-18T15:41:28.873830: step 535, loss 0.558553, acc 0.8\n",
      "2019-05-18T15:41:29.107570: step 540, loss 1.62561, acc 0.5\n",
      "2019-05-18T15:41:29.347449: step 545, loss 0.395527, acc 1\n",
      "2019-05-18T15:41:29.586104: step 550, loss 1.71424, acc 0.6\n",
      "2019-05-18T15:41:29.850418: step 555, loss 1.036, acc 0.7\n",
      "2019-05-18T15:41:30.093782: step 560, loss 1.14382, acc 0.8\n",
      "2019-05-18T15:41:30.355095: step 565, loss 1.43072, acc 0.6\n",
      "2019-05-18T15:41:30.571656: step 570, loss 3.21089, acc 0.4\n",
      "2019-05-18T15:41:30.834947: step 575, loss 1.77198, acc 0.6\n",
      "2019-05-18T15:41:31.073398: step 580, loss 1.73777, acc 0.5\n",
      "2019-05-18T15:41:31.316347: step 585, loss 1.46948, acc 0.8\n",
      "2019-05-18T15:41:31.558560: step 590, loss 1.11227, acc 0.8\n",
      "2019-05-18T15:41:31.797030: step 595, loss 2.10045, acc 0.6\n",
      "2019-05-18T15:41:32.049346: step 600, loss 0.756726, acc 0.9\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:32.486011: step 600, loss 0.983342, acc 0.802495\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0218077\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0218-600\n",
      "\n",
      "2019-05-18T15:41:33.299471: step 605, loss 1.482, acc 0.7\n",
      "2019-05-18T15:41:33.546015: step 610, loss 0.8297, acc 0.8\n",
      "2019-05-18T15:41:33.762953: step 615, loss 0.728005, acc 0.9\n",
      "2019-05-18T15:41:33.983814: step 620, loss 0.654266, acc 0.8\n",
      "2019-05-18T15:41:34.225502: step 625, loss 0.695691, acc 0.9\n",
      "2019-05-18T15:41:34.502567: step 630, loss 0.524735, acc 0.9\n",
      "2019-05-18T15:41:34.752898: step 635, loss 1.15696, acc 0.7\n",
      "2019-05-18T15:41:35.004186: step 640, loss 1.07271, acc 0.7\n",
      "2019-05-18T15:41:35.243004: step 645, loss 0.191517, acc 1\n",
      "2019-05-18T15:41:35.467147: step 650, loss 0.418583, acc 0.9\n",
      "2019-05-18T15:41:35.720829: step 655, loss 1.35477, acc 0.6\n",
      "2019-05-18T15:41:35.959614: step 660, loss 0.565011, acc 0.9\n",
      "2019-05-18T15:41:36.221510: step 665, loss 1.55012, acc 0.7\n",
      "2019-05-18T15:41:36.460425: step 670, loss 1.03592, acc 0.9\n",
      "2019-05-18T15:41:36.710913: step 675, loss 1.37649, acc 0.7\n",
      "2019-05-18T15:41:36.945717: step 680, loss 1.00562, acc 0.7\n",
      "2019-05-18T15:41:37.179234: step 685, loss 1.58017, acc 0.6\n",
      "2019-05-18T15:41:37.422728: step 690, loss 1.20234, acc 0.7\n",
      "2019-05-18T15:41:37.671058: step 695, loss 1.29175, acc 0.7\n",
      "2019-05-18T15:41:37.925371: step 700, loss 1.04439, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:38.311137: step 700, loss 0.967504, acc 0.7921\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0172588\n",
      "\n",
      "2019-05-18T15:41:38.572449: step 705, loss 1.46709, acc 0.7\n",
      "2019-05-18T15:41:38.811757: step 710, loss 1.72964, acc 0.6\n",
      "2019-05-18T15:41:39.045347: step 715, loss 0.808637, acc 0.8\n",
      "2019-05-18T15:41:39.291940: step 720, loss 0.527922, acc 0.9\n",
      "2019-05-18T15:41:39.526896: step 725, loss 0.485435, acc 0.9\n",
      "2019-05-18T15:41:39.787201: step 730, loss 1.23279, acc 0.8\n",
      "2019-05-18T15:41:40.031826: step 735, loss 2.12274, acc 0.7\n",
      "2019-05-18T15:41:40.287667: step 740, loss 0.973322, acc 0.8\n",
      "2019-05-18T15:41:40.521042: step 745, loss 2.1968, acc 0.7\n",
      "2019-05-18T15:41:40.750233: step 750, loss 1.35495, acc 0.7\n",
      "2019-05-18T15:41:40.994635: step 755, loss 1.17585, acc 0.8\n",
      "2019-05-18T15:41:41.228352: step 760, loss 0.788658, acc 0.9\n",
      "2019-05-18T15:41:41.469042: step 765, loss 1.49933, acc 0.7\n",
      "2019-05-18T15:41:41.707442: step 770, loss 0.998652, acc 0.7\n",
      "2019-05-18T15:41:41.963782: step 775, loss 1.18681, acc 0.8\n",
      "2019-05-18T15:41:42.190183: step 780, loss 1.0137, acc 0.7\n",
      "2019-05-18T15:41:42.453478: step 785, loss 0.340442, acc 1\n",
      "2019-05-18T15:41:42.719764: step 790, loss 1.10875, acc 0.8\n",
      "2019-05-18T15:41:42.960121: step 795, loss 0.722557, acc 0.9\n",
      "2019-05-18T15:41:43.231396: step 800, loss 0.825304, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:43.622961: step 800, loss 0.964992, acc 0.794179\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0165243\n",
      "\n",
      "2019-05-18T15:41:43.874138: step 805, loss 1.31651, acc 0.5\n",
      "2019-05-18T15:41:44.132484: step 810, loss 0.953728, acc 0.8\n",
      "2019-05-18T15:41:44.396646: step 815, loss 0.673758, acc 0.8\n",
      "2019-05-18T15:41:44.659213: step 820, loss 0.66228, acc 0.8\n",
      "2019-05-18T15:41:44.939782: step 825, loss 1.2834, acc 0.7\n",
      "2019-05-18T15:41:45.205355: step 830, loss 1.09725, acc 0.7\n",
      "2019-05-18T15:41:45.464201: step 835, loss 1.46945, acc 0.6\n",
      "2019-05-18T15:41:45.714531: step 840, loss 0.632482, acc 0.9\n",
      "2019-05-18T15:41:45.961379: step 845, loss 1.62596, acc 0.6\n",
      "2019-05-18T15:41:46.202761: step 850, loss 1.2706, acc 0.6\n",
      "2019-05-18T15:41:46.437450: step 855, loss 1.26668, acc 0.7\n",
      "2019-05-18T15:41:46.674725: step 860, loss 1.09147, acc 0.8\n",
      "2019-05-18T15:41:46.905129: step 865, loss 0.431775, acc 1\n",
      "2019-05-18T15:41:47.162422: step 870, loss 1.32649, acc 0.7\n",
      "2019-05-18T15:41:47.452813: step 875, loss 1.16519, acc 0.7\n",
      "2019-05-18T15:41:47.700631: step 880, loss 1.22194, acc 0.7\n",
      "2019-05-18T15:41:47.949918: step 885, loss 0.32031, acc 1\n",
      "2019-05-18T15:41:48.186309: step 890, loss 1.14738, acc 0.7\n",
      "2019-05-18T15:41:48.450605: step 895, loss 0.836199, acc 0.8\n",
      "2019-05-18T15:41:48.707914: step 900, loss 0.742101, acc 0.9\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:49.116779: step 900, loss 0.93615, acc 0.798337\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0202614\n",
      "\n",
      "2019-05-18T15:41:49.371416: step 905, loss 0.198841, acc 1\n",
      "2019-05-18T15:41:49.649616: step 910, loss 0.757914, acc 0.8\n",
      "2019-05-18T15:41:49.933181: step 915, loss 0.640497, acc 0.9\n",
      "2019-05-18T15:41:50.201276: step 920, loss 1.21655, acc 0.7\n",
      "2019-05-18T15:41:50.440946: step 925, loss 1.18725, acc 0.8\n",
      "2019-05-18T15:41:50.716034: step 930, loss 1.96271, acc 0.6\n",
      "2019-05-18T15:41:50.959740: step 935, loss 0.592588, acc 0.8\n",
      "2019-05-18T15:41:51.208122: step 940, loss 1.36122, acc 0.7\n",
      "2019-05-18T15:41:51.465181: step 945, loss 1.31182, acc 0.7\n",
      "2019-05-18T15:41:51.730490: step 950, loss 0.62727, acc 0.9\n",
      "2019-05-18T15:41:52.005820: step 955, loss 1.13937, acc 0.8\n",
      "2019-05-18T15:41:52.261819: step 960, loss 0.390785, acc 0.9\n",
      "2019-05-18T15:41:52.512298: step 965, loss 1.22524, acc 0.7\n",
      "2019-05-18T15:41:52.789069: step 970, loss 1.43898, acc 0.8\n",
      "2019-05-18T15:41:53.082229: step 975, loss 1.69461, acc 0.7\n",
      "2019-05-18T15:41:53.366856: step 980, loss 0.364385, acc 1\n",
      "2019-05-18T15:41:53.607473: step 985, loss 0.29622, acc 1\n",
      "2019-05-18T15:41:53.857632: step 990, loss 0.93816, acc 0.8\n",
      "2019-05-18T15:41:54.086110: step 995, loss 1.26118, acc 0.7\n",
      "2019-05-18T15:41:54.330368: step 1000, loss 0.583685, acc 0.9\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:41:54.729890: step 1000, loss 0.911673, acc 0.800416\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0243565\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0244-1000\n",
      "\n",
      "2019-05-18T15:41:55.440099: step 1005, loss 1.05698, acc 0.8\n",
      "2019-05-18T15:41:55.696415: step 1010, loss 1.03434, acc 0.8\n",
      "2019-05-18T15:41:55.923940: step 1015, loss 0.986684, acc 0.7\n",
      "2019-05-18T15:41:56.162913: step 1020, loss 0.328676, acc 1\n",
      "2019-05-18T15:41:56.402818: step 1025, loss 0.594503, acc 0.9\n",
      "2019-05-18T15:41:56.635275: step 1030, loss 0.418576, acc 0.9\n",
      "2019-05-18T15:41:56.882223: step 1035, loss 0.34116, acc 0.9\n",
      "2019-05-18T15:41:57.137890: step 1040, loss 0.618681, acc 0.8\n",
      "2019-05-18T15:41:57.387351: step 1045, loss 1.63869, acc 0.6\n",
      "2019-05-18T15:41:57.668476: step 1050, loss 0.786773, acc 0.8\n",
      "2019-05-18T15:41:57.926753: step 1055, loss 0.823317, acc 0.8\n",
      "2019-05-18T15:41:58.236368: step 1060, loss 1.56534, acc 0.7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-05-18T15:41:58.499165: step 1065, loss 0.675149, acc 0.9\n",
      "2019-05-18T15:41:58.755537: step 1070, loss 1.48391, acc 0.7\n",
      "2019-05-18T15:41:59.002900: step 1075, loss 1.71185, acc 0.6\n",
      "2019-05-18T15:41:59.213773: step 1080, loss 1.20818, acc 0.7\n",
      "2019-05-18T15:41:59.462109: step 1085, loss 1.48186, acc 0.4\n",
      "2019-05-18T15:41:59.707453: step 1090, loss 0.505844, acc 0.9\n",
      "2019-05-18T15:41:59.956304: step 1095, loss 1.43047, acc 0.8\n",
      "2019-05-18T15:42:00.222419: step 1100, loss 1.04392, acc 0.7\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:00.685862: step 1100, loss 0.930688, acc 0.798337\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0203169\n",
      "\n",
      "2019-05-18T15:42:00.923663: step 1105, loss 0.328514, acc 1\n",
      "2019-05-18T15:42:01.171002: step 1110, loss 0.909372, acc 0.8\n",
      "2019-05-18T15:42:01.410686: step 1115, loss 0.333004, acc 0.9\n",
      "2019-05-18T15:42:01.650719: step 1120, loss 0.825487, acc 0.9\n",
      "2019-05-18T15:42:01.894860: step 1125, loss 1.10445, acc 0.7\n",
      "2019-05-18T15:42:02.143119: step 1130, loss 1.45075, acc 0.6\n",
      "2019-05-18T15:42:02.375499: step 1135, loss 1.19894, acc 0.7\n",
      "2019-05-18T15:42:02.610105: step 1140, loss 0.744215, acc 0.8\n",
      "2019-05-18T15:42:02.895339: step 1145, loss 1.01719, acc 0.7\n",
      "2019-05-18T15:42:03.136979: step 1150, loss 0.649755, acc 0.8\n",
      "2019-05-18T15:42:03.392148: step 1155, loss 1.40814, acc 0.7\n",
      "2019-05-18T15:42:03.642415: step 1160, loss 0.506159, acc 0.8\n",
      "2019-05-18T15:42:03.864820: step 1165, loss 0.7927, acc 0.8\n",
      "2019-05-18T15:42:04.139791: step 1170, loss 0.203532, acc 1\n",
      "2019-05-18T15:42:04.386303: step 1175, loss 0.998907, acc 0.8\n",
      "2019-05-18T15:42:04.655113: step 1180, loss 1.35404, acc 0.8\n",
      "2019-05-18T15:42:04.937360: step 1185, loss 1.30765, acc 0.7\n",
      "2019-05-18T15:42:05.189252: step 1190, loss 0.222011, acc 1\n",
      "2019-05-18T15:42:05.440987: step 1195, loss 0.83324, acc 0.8\n",
      "2019-05-18T15:42:05.685349: step 1200, loss 0.450747, acc 1\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:06.067495: step 1200, loss 0.911617, acc 0.800416\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0262366\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0262-1200\n",
      "\n",
      "2019-05-18T15:42:06.862991: step 1205, loss 0.869941, acc 0.8\n",
      "2019-05-18T15:42:07.136261: step 1210, loss 0.852692, acc 0.8\n",
      "2019-05-18T15:42:07.385631: step 1215, loss 1.15571, acc 0.8\n",
      "2019-05-18T15:42:07.637401: step 1220, loss 0.598788, acc 0.8\n",
      "2019-05-18T15:42:07.908749: step 1225, loss 0.772339, acc 0.8\n",
      "2019-05-18T15:42:08.141138: step 1230, loss 0.979046, acc 0.7\n",
      "2019-05-18T15:42:08.387398: step 1235, loss 1.34515, acc 0.7\n",
      "2019-05-18T15:42:08.619892: step 1240, loss 1.49121, acc 0.4\n",
      "2019-05-18T15:42:08.855198: step 1245, loss 1.0056, acc 0.7\n",
      "2019-05-18T15:42:09.094963: step 1250, loss 0.830827, acc 0.8\n",
      "2019-05-18T15:42:09.318587: step 1255, loss 0.932852, acc 0.7\n",
      "2019-05-18T15:42:09.553491: step 1260, loss 0.797859, acc 0.9\n",
      "2019-05-18T15:42:09.781912: step 1265, loss 1.09244, acc 0.7\n",
      "2019-05-18T15:42:10.010613: step 1270, loss 1.33183, acc 0.7\n",
      "2019-05-18T15:42:10.229164: step 1275, loss 1.31246, acc 0.8\n",
      "2019-05-18T15:42:10.481710: step 1280, loss 1.03396, acc 0.7\n",
      "2019-05-18T15:42:10.719915: step 1285, loss 0.992071, acc 0.9\n",
      "2019-05-18T15:42:10.957217: step 1290, loss 0.59686, acc 0.9\n",
      "2019-05-18T15:42:11.183097: step 1295, loss 0.71543, acc 0.9\n",
      "2019-05-18T15:42:11.411206: step 1300, loss 0.80301, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:11.796711: step 1300, loss 0.892899, acc 0.806653\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0297396\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0297-1300\n",
      "\n",
      "2019-05-18T15:42:12.533830: step 1305, loss 1.23285, acc 0.8\n",
      "2019-05-18T15:42:12.778177: step 1310, loss 0.774019, acc 0.8\n",
      "2019-05-18T15:42:13.007914: step 1315, loss 0.418992, acc 0.9\n",
      "2019-05-18T15:42:13.246484: step 1320, loss 0.643757, acc 0.9\n",
      "2019-05-18T15:42:13.493957: step 1325, loss 1.01605, acc 0.8\n",
      "2019-05-18T15:42:13.738572: step 1330, loss 0.462962, acc 0.9\n",
      "2019-05-18T15:42:13.955684: step 1335, loss 0.676287, acc 0.8\n",
      "2019-05-18T15:42:14.186620: step 1340, loss 1.23071, acc 0.8\n",
      "2019-05-18T15:42:14.411598: step 1345, loss 0.661255, acc 0.8\n",
      "2019-05-18T15:42:14.647685: step 1350, loss 0.279509, acc 0.9\n",
      "2019-05-18T15:42:14.870620: step 1355, loss 0.233949, acc 1\n",
      "2019-05-18T15:42:15.096017: step 1360, loss 0.46083, acc 0.9\n",
      "2019-05-18T15:42:15.338375: step 1365, loss 0.964985, acc 0.9\n",
      "2019-05-18T15:42:15.567331: step 1370, loss 1.01259, acc 0.7\n",
      "2019-05-18T15:42:15.803007: step 1375, loss 0.667901, acc 0.9\n",
      "2019-05-18T15:42:16.040486: step 1380, loss 0.974526, acc 0.8\n",
      "2019-05-18T15:42:16.285238: step 1385, loss 0.698133, acc 0.9\n",
      "2019-05-18T15:42:16.539563: step 1390, loss 0.25548, acc 1\n",
      "2019-05-18T15:42:16.762816: step 1395, loss 1.54784, acc 0.5\n",
      "2019-05-18T15:42:16.986236: step 1400, loss 0.606418, acc 1\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:17.371757: step 1400, loss 0.872166, acc 0.794179\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0321252\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0321-1400\n",
      "\n",
      "2019-05-18T15:42:18.106797: step 1405, loss 0.341182, acc 0.9\n",
      "2019-05-18T15:42:18.327410: step 1410, loss 0.208943, acc 1\n",
      "2019-05-18T15:42:18.565786: step 1415, loss 0.773075, acc 0.8\n",
      "2019-05-18T15:42:18.786612: step 1420, loss 0.294935, acc 0.9\n",
      "2019-05-18T15:42:19.016003: step 1425, loss 1.29917, acc 0.7\n",
      "2019-05-18T15:42:19.238509: step 1430, loss 0.821932, acc 0.8\n",
      "2019-05-18T15:42:19.492842: step 1435, loss 1.28809, acc 0.7\n",
      "2019-05-18T15:42:19.724261: step 1440, loss 0.663997, acc 0.9\n",
      "2019-05-18T15:42:19.955157: step 1445, loss 1.51025, acc 0.6\n",
      "2019-05-18T15:42:20.183068: step 1450, loss 0.925869, acc 0.8\n",
      "2019-05-18T15:42:20.431521: step 1455, loss 0.623704, acc 0.7\n",
      "2019-05-18T15:42:20.665807: step 1460, loss 1.20861, acc 0.8\n",
      "2019-05-18T15:42:20.888003: step 1465, loss 0.792407, acc 0.9\n",
      "2019-05-18T15:42:21.115395: step 1470, loss 1.62366, acc 0.5\n",
      "2019-05-18T15:42:21.353743: step 1475, loss 0.356146, acc 1\n",
      "2019-05-18T15:42:21.582168: step 1480, loss 0.749439, acc 0.8\n",
      "2019-05-18T15:42:21.826104: step 1485, loss 0.784923, acc 0.8\n",
      "2019-05-18T15:42:22.076461: step 1490, loss 0.378437, acc 1\n",
      "2019-05-18T15:42:22.317338: step 1495, loss 1.33257, acc 0.6\n",
      "2019-05-18T15:42:22.552807: step 1500, loss 0.749226, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:22.920002: step 1500, loss 0.914393, acc 0.798337\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.032643\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0326-1500\n",
      "\n",
      "2019-05-18T15:42:23.583116: step 1505, loss 1.01766, acc 0.8\n",
      "2019-05-18T15:42:23.812624: step 1510, loss 0.876184, acc 0.7\n",
      "2019-05-18T15:42:24.054003: step 1515, loss 0.612255, acc 0.8\n",
      "2019-05-18T15:42:24.293391: step 1520, loss 0.713847, acc 0.9\n",
      "2019-05-18T15:42:24.523307: step 1525, loss 0.590477, acc 0.9\n",
      "2019-05-18T15:42:24.760829: step 1530, loss 0.53858, acc 0.9\n",
      "2019-05-18T15:42:24.995962: step 1535, loss 1.67065, acc 0.7\n",
      "2019-05-18T15:42:25.234411: step 1540, loss 2.13164, acc 0.5\n",
      "2019-05-18T15:42:25.471683: step 1545, loss 0.942507, acc 0.8\n",
      "2019-05-18T15:42:25.712554: step 1550, loss 1.05327, acc 0.7\n",
      "2019-05-18T15:42:25.957898: step 1555, loss 1.68197, acc 0.7\n",
      "2019-05-18T15:42:26.203387: step 1560, loss 0.989686, acc 0.7\n",
      "2019-05-18T15:42:26.426787: step 1565, loss 0.668469, acc 0.9\n",
      "2019-05-18T15:42:26.651188: step 1570, loss 0.771656, acc 0.9\n",
      "2019-05-18T15:42:26.871794: step 1575, loss 1.25901, acc 0.8\n",
      "2019-05-18T15:42:27.113656: step 1580, loss 0.680106, acc 0.8\n",
      "2019-05-18T15:42:27.342028: step 1585, loss 1.43928, acc 0.6\n",
      "2019-05-18T15:42:27.575651: step 1590, loss 0.917241, acc 0.8\n",
      "2019-05-18T15:42:27.819747: step 1595, loss 1.33714, acc 0.7\n",
      "2019-05-18T15:42:28.080052: step 1600, loss 1.36265, acc 0.6\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:28.487913: step 1600, loss 0.867094, acc 0.804574\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0333353\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0333-1600\n",
      "\n",
      "2019-05-18T15:42:29.263627: step 1605, loss 0.592621, acc 0.8\n",
      "2019-05-18T15:42:29.514274: step 1610, loss 0.491172, acc 0.9\n",
      "2019-05-18T15:42:29.768738: step 1615, loss 0.817733, acc 0.9\n",
      "2019-05-18T15:42:30.042006: step 1620, loss 1.40799, acc 0.7\n",
      "2019-05-18T15:42:30.292440: step 1625, loss 0.811716, acc 0.8\n",
      "2019-05-18T15:42:30.535800: step 1630, loss 0.246832, acc 1\n",
      "2019-05-18T15:42:30.763183: step 1635, loss 0.840492, acc 0.7\n",
      "2019-05-18T15:42:30.998782: step 1640, loss 0.558978, acc 0.8\n",
      "2019-05-18T15:42:31.239130: step 1645, loss 0.969361, acc 0.8\n",
      "2019-05-18T15:42:31.543254: step 1650, loss 0.691575, acc 0.8\n",
      "2019-05-18T15:42:31.830141: step 1655, loss 0.572422, acc 0.7\n",
      "2019-05-18T15:42:32.087050: step 1660, loss 0.953631, acc 0.8\n",
      "2019-05-18T15:42:32.343556: step 1665, loss 0.291146, acc 1\n",
      "2019-05-18T15:42:32.616811: step 1670, loss 0.272762, acc 1\n",
      "2019-05-18T15:42:32.903176: step 1675, loss 1.0478, acc 0.8\n",
      "2019-05-18T15:42:33.165747: step 1680, loss 0.935412, acc 0.7\n",
      "2019-05-18T15:42:33.415395: step 1685, loss 0.76513, acc 0.7\n",
      "2019-05-18T15:42:33.658258: step 1690, loss 1.45304, acc 0.7\n",
      "2019-05-18T15:42:33.907898: step 1695, loss 0.569631, acc 1\n",
      "2019-05-18T15:42:34.151492: step 1700, loss 0.611338, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:34.561342: step 1700, loss 0.86437, acc 0.808732\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0518022\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0518-1700\n",
      "\n",
      "2019-05-18T15:42:35.348599: step 1705, loss 0.869287, acc 0.8\n",
      "2019-05-18T15:42:35.592960: step 1710, loss 0.771945, acc 0.9\n",
      "2019-05-18T15:42:35.844350: step 1715, loss 1.02553, acc 0.7\n",
      "2019-05-18T15:42:36.090066: step 1720, loss 0.514338, acc 0.9\n",
      "2019-05-18T15:42:36.331730: step 1725, loss 0.254048, acc 1\n",
      "2019-05-18T15:42:36.574081: step 1730, loss 1.00008, acc 0.8\n",
      "2019-05-18T15:42:36.814507: step 1735, loss 1.33229, acc 0.7\n",
      "2019-05-18T15:42:37.051062: step 1740, loss 1.06978, acc 0.8\n",
      "2019-05-18T15:42:37.289321: step 1745, loss 0.735101, acc 0.8\n",
      "2019-05-18T15:42:37.517814: step 1750, loss 0.986088, acc 0.8\n",
      "2019-05-18T15:42:37.737588: step 1755, loss 0.795019, acc 0.9\n",
      "2019-05-18T15:42:37.990001: step 1760, loss 0.83275, acc 0.8\n",
      "2019-05-18T15:42:38.229414: step 1765, loss 0.857158, acc 0.8\n",
      "2019-05-18T15:42:38.470968: step 1770, loss 0.873421, acc 0.9\n",
      "2019-05-18T15:42:38.709647: step 1775, loss 0.994523, acc 0.7\n",
      "2019-05-18T15:42:38.934787: step 1780, loss 0.573468, acc 0.9\n",
      "2019-05-18T15:42:39.171397: step 1785, loss 0.442277, acc 0.9\n",
      "2019-05-18T15:42:39.419252: step 1790, loss 0.466571, acc 0.9\n",
      "2019-05-18T15:42:39.669849: step 1795, loss 0.489047, acc 0.9\n",
      "2019-05-18T15:42:39.930460: step 1800, loss 0.562264, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:40.324870: step 1800, loss 0.856908, acc 0.81289\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0873716\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0874-1800\n",
      "\n",
      "2019-05-18T15:42:41.068685: step 1805, loss 0.183142, acc 1\n",
      "2019-05-18T15:42:41.317018: step 1810, loss 1.32912, acc 0.6\n",
      "2019-05-18T15:42:41.584301: step 1815, loss 0.944073, acc 0.7\n",
      "2019-05-18T15:42:41.834712: step 1820, loss 1.1158, acc 0.7\n",
      "2019-05-18T15:42:42.067941: step 1825, loss 0.77291, acc 0.8\n",
      "2019-05-18T15:42:42.328246: step 1830, loss 0.495248, acc 0.8\n",
      "2019-05-18T15:42:42.605467: step 1835, loss 0.441887, acc 0.9\n",
      "2019-05-18T15:42:42.837452: step 1840, loss 0.705622, acc 0.8\n",
      "2019-05-18T15:42:43.072824: step 1845, loss 1.23274, acc 0.8\n",
      "2019-05-18T15:42:43.319568: step 1850, loss 0.194756, acc 1\n",
      "2019-05-18T15:42:43.562693: step 1855, loss 0.882749, acc 0.8\n",
      "2019-05-18T15:42:43.794842: step 1860, loss 0.665572, acc 0.9\n",
      "2019-05-18T15:42:44.039015: step 1865, loss 0.649329, acc 0.8\n",
      "2019-05-18T15:42:44.282366: step 1870, loss 0.356188, acc 0.9\n",
      "2019-05-18T15:42:44.547766: step 1875, loss 0.749891, acc 0.8\n",
      "2019-05-18T15:42:44.822923: step 1880, loss 0.926002, acc 0.8\n",
      "2019-05-18T15:42:45.061470: step 1885, loss 1.39841, acc 0.7\n",
      "2019-05-18T15:42:45.331929: step 1890, loss 0.273012, acc 1\n",
      "2019-05-18T15:42:45.575278: step 1895, loss 0.296603, acc 1\n",
      "2019-05-18T15:42:45.822049: step 1900, loss 0.693667, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:46.220747: step 1900, loss 0.837452, acc 0.827443\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0911169\n",
      "\n",
      "Saved model checkpoint to E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0911-1900\n",
      "\n",
      "2019-05-18T15:42:46.954137: step 1905, loss 2.38158, acc 0.6\n",
      "2019-05-18T15:42:47.186514: step 1910, loss 0.828901, acc 0.7\n",
      "2019-05-18T15:42:47.426836: step 1915, loss 0.796876, acc 0.8\n",
      "2019-05-18T15:42:47.664194: step 1920, loss 0.220678, acc 1\n",
      "2019-05-18T15:42:47.917514: step 1925, loss 0.404716, acc 1\n",
      "2019-05-18T15:42:48.151155: step 1930, loss 0.948949, acc 0.8\n",
      "2019-05-18T15:42:48.378869: step 1935, loss 1.41817, acc 0.6\n",
      "2019-05-18T15:42:48.606319: step 1940, loss 1.14912, acc 0.8\n",
      "2019-05-18T15:42:48.870618: step 1945, loss 1.38712, acc 0.6\n",
      "2019-05-18T15:42:49.131383: step 1950, loss 1.14893, acc 0.7\n",
      "2019-05-18T15:42:49.369618: step 1955, loss 0.603775, acc 0.9\n",
      "2019-05-18T15:42:49.598676: step 1960, loss 1.11603, acc 0.6\n",
      "2019-05-18T15:42:49.846969: step 1965, loss 0.71862, acc 0.7\n",
      "2019-05-18T15:42:50.085820: step 1970, loss 0.459467, acc 0.9\n",
      "2019-05-18T15:42:50.326248: step 1975, loss 1.04657, acc 0.9\n",
      "2019-05-18T15:42:50.579494: step 1980, loss 1.14114, acc 0.8\n",
      "2019-05-18T15:42:50.837803: step 1985, loss 0.545936, acc 0.9\n",
      "2019-05-18T15:42:51.071182: step 1990, loss 0.446075, acc 0.9\n",
      "2019-05-18T15:42:51.316058: step 1995, loss 1.2536, acc 0.7\n",
      "2019-05-18T15:42:51.552405: step 2000, loss 0.722435, acc 0.8\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:51.947972: step 2000, loss 0.840053, acc 0.825364\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0891466\n",
      "\n",
      "2019-05-18T15:42:52.172751: step 2005, loss 0.732182, acc 0.9\n",
      "2019-05-18T15:42:52.406686: step 2010, loss 0.818589, acc 0.8\n",
      "2019-05-18T15:42:52.644291: step 2015, loss 0.804793, acc 0.8\n",
      "2019-05-18T15:42:52.886620: step 2020, loss 1.25844, acc 0.7\n",
      "2019-05-18T15:42:53.119996: step 2025, loss 0.880522, acc 0.9\n",
      "2019-05-18T15:42:53.364879: step 2030, loss 0.807216, acc 0.8\n",
      "2019-05-18T15:42:53.598929: step 2035, loss 1.23368, acc 0.8\n",
      "2019-05-18T15:42:53.848504: step 2040, loss 1.39874, acc 0.5\n",
      "2019-05-18T15:42:54.088058: step 2045, loss 0.563114, acc 0.9\n",
      "2019-05-18T15:42:54.341814: step 2050, loss 1.65886, acc 0.8\n",
      "2019-05-18T15:42:54.584587: step 2055, loss 0.835327, acc 0.8\n",
      "2019-05-18T15:42:54.824118: step 2060, loss 0.866258, acc 0.9\n",
      "2019-05-18T15:42:55.063769: step 2065, loss 0.419939, acc 1\n",
      "2019-05-18T15:42:55.296594: step 2070, loss 0.444969, acc 0.8\n",
      "2019-05-18T15:42:55.533247: step 2075, loss 1.27361, acc 0.7\n",
      "2019-05-18T15:42:55.765652: step 2080, loss 0.656912, acc 0.9\n",
      "2019-05-18T15:42:55.985499: step 2085, loss 0.636288, acc 0.9\n",
      "2019-05-18T15:42:56.220451: step 2090, loss 0.680548, acc 0.9\n",
      "2019-05-18T15:42:56.489743: step 2095, loss 0.286545, acc 0.9\n",
      "2019-05-18T15:42:56.722121: step 2100, loss 0.372464, acc 0.9\n",
      "\n",
      "Evaluation:\n",
      "2019-05-18T15:42:57.098851: step 2100, loss 0.828979, acc 0.7921\n",
      "[UNOFFICIAL] (2*9+1)-Way Macro-Average F1 Score (excluding Other): 0.0659817\n",
      "\n",
      "2019-05-18T15:42:57.365142: step 2105, loss 0.754271, acc 0.8\n",
      "2019-05-18T15:42:57.602035: step 2110, loss 0.444686, acc 0.9\n",
      "2019-05-18T15:42:57.870332: step 2115, loss 0.492527, acc 0.9\n",
      "2019-05-18T15:42:58.122714: step 2120, loss 0.493057, acc 0.9\n",
      "2019-05-18T15:42:58.362516: step 2125, loss 1.25762, acc 0.7\n",
      "2019-05-18T15:42:58.605923: step 2130, loss 0.751594, acc 0.8\n",
      "2019-05-18T15:42:58.853278: step 2135, loss 0.786725, acc 0.9\n",
      "2019-05-18T15:42:59.105728: step 2140, loss 0.197537, acc 1\n",
      "2019-05-18T15:42:59.360173: step 2145, loss 0.403657, acc 0.9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-05-18T15:42:59.608096: step 2150, loss 0.324538, acc 0.9\n",
      "2019-05-18T15:42:59.859538: step 2155, loss 0.408016, acc 1\n",
      "2019-05-18T15:43:00.115736: step 2160, loss 0.789831, acc 0.7\n",
      "2019-05-18T15:43:00.360092: step 2165, loss 0.867749, acc 0.8\n",
      "2019-05-18T15:43:00.619250: step 2170, loss 0.467613, acc 0.857143\n"
     ]
    }
   ],
   "source": [
    "train(settings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6、模型评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval(settings):\n",
    "    print(\"---eval---\")\n",
    "    with tf.device('/cpu:0'):\n",
    "        x_text, y = load_data_and_labels_cn(path=settings.test_path,settings=settings)\n",
    "\n",
    "    print(\"len(x_text):{0}\".format(len(x_text)))\n",
    "    print(\"len(y):{0}\".format(len(y)))\n",
    "    \n",
    "    \n",
    "    # Map data into vocabulary\n",
    "    text_path = os.path.join(settings.checkpoint_dir, \"..\", \"vocab\")\n",
    "    text_vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor.restore(text_path)\n",
    "    x = np.array(list(text_vocab_processor.transform(x_text)))\n",
    "\n",
    "    checkpoint_file = tf.train.latest_checkpoint(settings.checkpoint_dir)\n",
    "\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        session_conf = tf.ConfigProto(\n",
    "            allow_soft_placement=settings.allow_soft_placement,\n",
    "            log_device_placement=settings.log_device_placement)\n",
    "        session_conf.gpu_options.allow_growth = settings.gpu_allow_growth\n",
    "        sess = tf.Session(config=session_conf)\n",
    "        with sess.as_default():\n",
    "            # Load the saved meta graph and restore variables\n",
    "            saver = tf.train.import_meta_graph(\"{}.meta\".format(checkpoint_file))\n",
    "            saver.restore(sess, checkpoint_file)\n",
    "\n",
    "            # Get the placeholders from the graph by name\n",
    "            input_text = graph.get_operation_by_name(\"input_text\").outputs[0]\n",
    "            # input_y = graph.get_operation_by_name(\"input_y\").outputs[0]\n",
    "            emb_dropout_keep_prob = graph.get_operation_by_name(\"emb_dropout_keep_prob\").outputs[0]\n",
    "            rnn_dropout_keep_prob = graph.get_operation_by_name(\"rnn_dropout_keep_prob\").outputs[0]\n",
    "            dropout_keep_prob = graph.get_operation_by_name(\"dropout_keep_prob\").outputs[0]\n",
    "\n",
    "            # Tensors we want to evaluate\n",
    "            predictions = graph.get_operation_by_name(\"output/predictions\").outputs[0]\n",
    "\n",
    "            # Generate batches for one epoch\n",
    "            batches = batch_iter(list(x), settings.batch_size, 1, shuffle=False)\n",
    "\n",
    "            # Collect the predictions here\n",
    "            preds = []\n",
    "            for x_batch in batches:\n",
    "                pred = sess.run(predictions, {input_text: x_batch,\n",
    "                                              emb_dropout_keep_prob: 1.0,\n",
    "                                              rnn_dropout_keep_prob: 1.0,\n",
    "                                              dropout_keep_prob: 1.0})\n",
    "                preds.append(pred)\n",
    "            preds = np.concatenate(preds)\n",
    "            truths = np.argmax(y, axis=1)\n",
    "\n",
    "            relation2id,id2relation=load_relation2id_file_cn(filename=settings.relation2id_path)   \n",
    "            prediction_path = os.path.join(settings.checkpoint_dir, \"..\", \"predictions.txt\")\n",
    "            predictionClass_path = os.path.join(settings.checkpoint_dir, \"..\", \"predictionsClass.txt\")\n",
    "            truth_path = os.path.join(settings.checkpoint_dir, \"..\", \"ground_truths.txt\")\n",
    "            prediction_file = open_file(prediction_path, 'w')\n",
    "            predictionClass_file = open_file(predictionClass_path, 'w')\n",
    "            truth_file = open_file(truth_path, 'w')\n",
    "            for i in range(len(preds)):\n",
    "                prediction_file.write(\"{}\\t{}\\n\".format(i, id2relation[preds[i]]))\n",
    "                predictionClass_file.write(\"{0}\\t{1}\\n\".format(i, preds[i]))\n",
    "                truth_file.write(\"{}\\t{}\\n\".format(i, id2relation[truths[i]]))\n",
    "            prediction_file.close()\n",
    "            truth_file.close()\n",
    "\n",
    "#             perl_path = os.path.join(os.path.curdir,\n",
    "#                                      \"SemEval2010_task8_all_data\",\n",
    "#                                      \"SemEval2010_task8_scorer-v1.2\",\n",
    "#                                      \"semeval2010_task8_scorer-v1.2.pl\")\n",
    "#             process = subprocess.Popen([\"perl\", perl_path, prediction_path, truth_path], stdout=subprocess.PIPE)\n",
    "#             for line in str(process.communicate()[0].decode(\"utf-8\")).split(\"\\\\n\"):\n",
    "#                 print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---eval---\n",
      "labels_flat:[0 0 0 ... 0 0 0]\n",
      "labels_count:1\n",
      "len(x_text):77092\n",
      "len(y):77092\n",
      "INFO:tensorflow:Restoring parameters from E:\\pythonWp\\nlp\\relation_extraction\\relation_extraction_study\\Attention-Based-BiLSTM-relation-extraction\\runs\\1558165255\\checkpoints\\model-0.0911-1900\n"
     ]
    }
   ],
   "source": [
    "settings.checkpoint_dir = \"runs/1558165255/checkpoints/\"\n",
    "settings.test_path=\"E:/pythonWp/game/CCKS2019/RelationshipExtraction/origin_data/test_f_char.txt\"\n",
    "eval(settings=settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
